{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aef8f7ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import time\n",
    "import openai\n",
    "import backoff\n",
    "import os\n",
    "import json\n",
    "from tqdm import tqdm\n",
    "from random import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f4cbc150",
   "metadata": {},
   "outputs": [],
   "source": [
    "API_KEY = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "39157a75",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_now(return_flag=0):\n",
    "    t_delta = datetime.timedelta(hours=9)\n",
    "    JST = datetime.timezone(t_delta, 'JST')\n",
    "    now = datetime.datetime.now(JST)\n",
    "    now = now.strftime('%Y/%m/%d %H:%M:%S')\n",
    "    if return_flag == 0:\n",
    "        print(now)\n",
    "    elif return_flag == 1:\n",
    "        return now\n",
    "    else:\n",
    "        pass\n",
    "\n",
    "@backoff.on_exception(backoff.expo, openai.error.RateLimitError, max_tries=8)\n",
    "def completions_with_backoff(**kwargs):\n",
    "    return openai.Completion.create(**kwargs)\n",
    "\n",
    "\n",
    "def call_gpt35_turbo(prompt):\n",
    "    messages = [{\"role\": \"user\", \"content\": prompt}]\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=\"gpt-3.5-turbo\",\n",
    "        messages=messages,\n",
    "    )\n",
    "    response_message = response[\"choices\"][0][\"message\"][\"content\"]\n",
    "    return response_message\n",
    "\n",
    "def call_gpt4_turbo(prompt):\n",
    "    messages = [{\"role\": \"user\", \"content\": prompt}]\n",
    "    response = openai.ChatCompletion.create(\n",
    "        model=\"gpt-4\",\n",
    "        messages=messages,\n",
    "    )\n",
    "    response_message = response[\"choices\"][0][\"message\"][\"content\"]\n",
    "    return response_message\n",
    "\n",
    "# Sentence Generator (Decoder) for GPT-3 ...\n",
    "def decoder_for_openai(mode_name, input):\n",
    "    \n",
    "    # GPT-3 API allows each users execute the API within 60 times in a minute ...\n",
    "    # time.sleep(1)\n",
    "    time.sleep(1)\n",
    "    \n",
    "    # https://beta.openai.com/account/api-keys\n",
    "    # openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "    openai.api_key = API_KEY\n",
    "    #print(openai.api_key)\n",
    "\n",
    "    if mode_name == \"gpt3.5-turbo\":\n",
    "        return call_gpt35_turbo(input)\n",
    "    elif mode_name == \"gpt-4\":\n",
    "        return call_gpt4_turbo(input)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7e56d60",
   "metadata": {},
   "source": [
    "# 读取instruction graph测试集\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "db73274d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 1933/1933 [00:00<00:00, 28702.18it/s]\n"
     ]
    }
   ],
   "source": [
    "def read_instruction_graph(data_dir):\n",
    "    with open(os.path.join(data_dir, \"instructgraph_test_mini_data.json\"), \"r\", encoding=\"utf-8\") as fr:\n",
    "        instructgraph_test_data = [json.loads(i) for i in tqdm(fr.readlines())]\n",
    "    return instructgraph_test_data\n",
    "data_dir = \"/Users/wangjianing/Desktop/项目文件/科研项目/LLM+Graph/InstructGraph/data/instruction_dataset/released/\"\n",
    "instructgraph_test_data = read_instruction_graph(data_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "77132845",
   "metadata": {},
   "source": [
    "# GPT-3.5实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "78046749",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = \"/Users/wangjianing/Desktop/项目文件/科研项目/LLM+Graph/InstructGraph/output/openai/gpt3.5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "5ba1e86f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████| 1946/1946 [00:01<00:00, 1088.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "have predicted 1946 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 读取已经保存好的预测记录\n",
    "def read_saved_predictions(output_dir):\n",
    "    if not os.path.exists(os.path.join(output_dir, \"all_prediction.json\")):\n",
    "        return list(), list()\n",
    "    with open(os.path.join(output_dir, \"all_prediction.json\"), \"r\", encoding=\"utf-8\") as fr:\n",
    "        examples = [json.loads(i) for i in tqdm(fr.readlines())]\n",
    "    saved_name = list()\n",
    "    saved_example_list = list()\n",
    "    for example in examples:\n",
    "        task_name = example[\"task_name\"]\n",
    "        idx = example[\"idx\"]\n",
    "        saved_example_list.append(example)\n",
    "        saved_name.append(\"{}-{}\".format(task_name, idx))\n",
    "    return saved_example_list, saved_name\n",
    "\n",
    "prediction_example_list = list()\n",
    "saved_example_list, saved_name = read_saved_predictions(output_dir)\n",
    "prediction_example_list.extend(saved_example_list)\n",
    "print(\"have predicted {} examples\".format(len(prediction_example_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "0964a128",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████| 1933/1933 [1:41:31<00:00,  3.15s/it]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Step2: 调用chatgpt获得cot，每5次调用保存一下。\n",
    "注意：如果该行程序被终止，请务必从Step1开始重新执行\n",
    "\"\"\"\n",
    "call_num = 0\n",
    "for example in tqdm(instructgraph_test_data):\n",
    "    \n",
    "    task_name = example[\"task_name\"]\n",
    "    idx = example[\"idx\"]\n",
    "    \n",
    "    if \"{}-{}\".format(task_name, str(idx)) in saved_example_list:\n",
    "        continue\n",
    "    \n",
    "    answer = example[\"answer\"]\n",
    "    instruction = example[\"instruction\"]\n",
    "    \n",
    "    # 调用chatgpt获得推理结果\n",
    "    response = decoder_for_openai(\"gpt3.5-turbo\", instruction)\n",
    "    call_num += 1\n",
    "    # 先生成，后续再单独判断推理的结果是否与标签一致\n",
    "    example[\"prediction\"] = response\n",
    "    prediction_example_list.append(example)\n",
    "    # 每5次成功调用就保存一次文件，避免丢失\n",
    "    if call_num % 5 == 0:\n",
    "#         with open(os.path.join(output_dir, \"all_predictions.json\"), \"w\", encoding=\"utf-8\") as fw:\n",
    "#             for example in prediction_example_list:\n",
    "#                 fw.write(json.dumps(example) + \"\\n\")\n",
    "        with open(os.path.join(output_dir, \"all_prediction.json\"), \"w\", encoding=\"utf-8\") as fw:\n",
    "            json.dump(prediction_example_list, fw, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89204bff",
   "metadata": {},
   "source": [
    "# GPT-4实验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "211447c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = \"/Users/wangjianing/Desktop/项目文件/科研项目/LLM+Graph/InstructGraph/output/openai/gpt4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "17a1f23e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 550/550 [00:00<00:00, 3076.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "have predicted 550 examples\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 读取已经保存好的预测记录\n",
    "def read_saved_predictions(output_dir):\n",
    "    if not os.path.exists(os.path.join(output_dir, \"all_predictions.json\")):\n",
    "        return list(), list()\n",
    "    with open(os.path.join(output_dir, \"all_predictions.json\"), \"r\", encoding=\"utf-8\") as fr:\n",
    "        examples = [json.loads(i) for i in tqdm(fr.readlines())]\n",
    "    saved_name = list()\n",
    "    saved_example_list = list()\n",
    "    for example in examples:\n",
    "        task_name = example[\"task_name\"]\n",
    "        idx = example[\"idx\"]\n",
    "        saved_example_list.append(example)\n",
    "        saved_name.append(\"{}-{}\".format(task_name, idx))\n",
    "    return saved_example_list, saved_name\n",
    "\n",
    "prediction_example_list = list()\n",
    "saved_example_list, saved_name = read_saved_predictions(output_dir)\n",
    "prediction_example_list.extend(saved_example_list)\n",
    "print(\"have predicted {} examples\".format(len(prediction_example_list)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db671f80",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a2bf2676",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'graph-language-modeling-graph-node-cls-pubmed-1013'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(prediction_example_list)\n",
    "prediction_example_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "efae56f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████▍                            | 550/1933 [00:13<00:32, 42.19it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[0;32mIn [61]\u001b[0m, in \u001b[0;36m<cell line: 6>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     15\u001b[0m instruction \u001b[38;5;241m=\u001b[39m example[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124minstruction\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     17\u001b[0m \u001b[38;5;66;03m# 调用chatgpt获得推理结果\u001b[39;00m\n\u001b[0;32m---> 18\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mdecoder_for_openai\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgpt-4\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minstruction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     19\u001b[0m call_num \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;66;03m# 先生成，后续再单独判断推理的结果是否与标签一致\u001b[39;00m\n",
      "Input \u001b[0;32mIn [29]\u001b[0m, in \u001b[0;36mdecoder_for_openai\u001b[0;34m(mode_name, input)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m call_gpt35_turbo(\u001b[38;5;28minput\u001b[39m)\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode_name \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgpt-4\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m---> 51\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall_gpt4_turbo\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [29]\u001b[0m, in \u001b[0;36mcall_gpt4_turbo\u001b[0;34m(prompt)\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcall_gpt4_turbo\u001b[39m(prompt):\n\u001b[1;32m     28\u001b[0m     messages \u001b[38;5;241m=\u001b[39m [{\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124muser\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: prompt}]\n\u001b[0;32m---> 29\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[43mopenai\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mChatCompletion\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     30\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mgpt-4\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m     31\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     32\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     33\u001b[0m     response_message \u001b[38;5;241m=\u001b[39m response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchoices\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;241m0\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessage\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m response_message\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/openai/api_resources/chat_completion.py:25\u001b[0m, in \u001b[0;36mChatCompletion.create\u001b[0;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m     24\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 25\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     26\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m TryAgain \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     27\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m timeout \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m time\u001b[38;5;241m.\u001b[39mtime() \u001b[38;5;241m>\u001b[39m start \u001b[38;5;241m+\u001b[39m timeout:\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/openai/api_resources/abstract/engine_api_resource.py:153\u001b[0m, in \u001b[0;36mEngineAPIResource.create\u001b[0;34m(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate\u001b[39m(\n\u001b[1;32m    129\u001b[0m     \u001b[38;5;28mcls\u001b[39m,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    136\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mparams,\n\u001b[1;32m    137\u001b[0m ):\n\u001b[1;32m    138\u001b[0m     (\n\u001b[1;32m    139\u001b[0m         deployment_id,\n\u001b[1;32m    140\u001b[0m         engine,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    150\u001b[0m         api_key, api_base, api_type, api_version, organization, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mparams\n\u001b[1;32m    151\u001b[0m     )\n\u001b[0;32m--> 153\u001b[0m     response, _, api_key \u001b[38;5;241m=\u001b[39m \u001b[43mrequestor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    154\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m    155\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    156\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    157\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    158\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    159\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    160\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    161\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    163\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m stream:\n\u001b[1;32m    164\u001b[0m         \u001b[38;5;66;03m# must be an iterator\u001b[39;00m\n\u001b[1;32m    165\u001b[0m         \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(response, OpenAIResponse)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/openai/api_requestor.py:216\u001b[0m, in \u001b[0;36mAPIRequestor.request\u001b[0;34m(self, method, url, params, headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[1;32m    205\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrequest\u001b[39m(\n\u001b[1;32m    206\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    207\u001b[0m     method,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    214\u001b[0m     request_timeout: Optional[Union[\u001b[38;5;28mfloat\u001b[39m, Tuple[\u001b[38;5;28mfloat\u001b[39m, \u001b[38;5;28mfloat\u001b[39m]]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    215\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Union[OpenAIResponse, Iterator[OpenAIResponse]], \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mstr\u001b[39m]:\n\u001b[0;32m--> 216\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_raw\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    217\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlower\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    218\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    219\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparams\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    220\u001b[0m \u001b[43m        \u001b[49m\u001b[43msupplied_headers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    221\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    222\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    223\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    224\u001b[0m \u001b[43m        \u001b[49m\u001b[43mrequest_timeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_timeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    225\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    226\u001b[0m     resp, got_stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_interpret_response(result, stream)\n\u001b[1;32m    227\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m resp, got_stream, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mapi_key\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/openai/api_requestor.py:516\u001b[0m, in \u001b[0;36mAPIRequestor.request_raw\u001b[0;34m(self, method, url, params, supplied_headers, files, stream, request_id, request_timeout)\u001b[0m\n\u001b[1;32m    514\u001b[0m     _thread_context\u001b[38;5;241m.\u001b[39msession \u001b[38;5;241m=\u001b[39m _make_session()\n\u001b[1;32m    515\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 516\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[43m_thread_context\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    517\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    518\u001b[0m \u001b[43m        \u001b[49m\u001b[43mabs_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    519\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    520\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    521\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfiles\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfiles\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    522\u001b[0m \u001b[43m        \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    523\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest_timeout\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mif\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequest_timeout\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01melse\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mTIMEOUT_SECS\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    524\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    525\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mTimeout \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    526\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m error\u001b[38;5;241m.\u001b[39mTimeout(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRequest timed out: \u001b[39m\u001b[38;5;132;01m{}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(e)) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    585\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    587\u001b[0m }\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/requests/adapters.py:486\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    483\u001b[0m     timeout \u001b[38;5;241m=\u001b[39m TimeoutSauce(connect\u001b[38;5;241m=\u001b[39mtimeout, read\u001b[38;5;241m=\u001b[39mtimeout)\n\u001b[1;32m    485\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 486\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    487\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    488\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    489\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    490\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    491\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    492\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    496\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[1;32m    501\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(err, request\u001b[38;5;241m=\u001b[39mrequest)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/urllib3/connectionpool.py:703\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m    700\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_prepare_proxy(conn)\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m httplib_response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    704\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    706\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    707\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    708\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    709\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    710\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    711\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    713\u001b[0m \u001b[38;5;66;03m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m    714\u001b[0m \u001b[38;5;66;03m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m    715\u001b[0m \u001b[38;5;66;03m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m    716\u001b[0m \u001b[38;5;66;03m# mess.\u001b[39;00m\n\u001b[1;32m    717\u001b[0m response_conn \u001b[38;5;241m=\u001b[39m conn \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m release_conn \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/urllib3/connectionpool.py:449\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    444\u001b[0m             httplib_response \u001b[38;5;241m=\u001b[39m conn\u001b[38;5;241m.\u001b[39mgetresponse()\n\u001b[1;32m    445\u001b[0m         \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m             \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m             \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m             \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[0;32m--> 449\u001b[0m             \u001b[43msix\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_from\u001b[49m\u001b[43m(\u001b[49m\u001b[43me\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (SocketTimeout, BaseSSLError, SocketError) \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    451\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_raise_timeout(err\u001b[38;5;241m=\u001b[39me, url\u001b[38;5;241m=\u001b[39murl, timeout_value\u001b[38;5;241m=\u001b[39mread_timeout)\n",
      "File \u001b[0;32m<string>:3\u001b[0m, in \u001b[0;36mraise_from\u001b[0;34m(value, from_value)\u001b[0m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/site-packages/urllib3/connectionpool.py:444\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    441\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mTypeError\u001b[39;00m:\n\u001b[1;32m    442\u001b[0m     \u001b[38;5;66;03m# Python 3\u001b[39;00m\n\u001b[1;32m    443\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 444\u001b[0m         httplib_response \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgetresponse\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    445\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mBaseException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    446\u001b[0m         \u001b[38;5;66;03m# Remove the TypeError from the exception chain in\u001b[39;00m\n\u001b[1;32m    447\u001b[0m         \u001b[38;5;66;03m# Python 3 (including for exceptions like SystemExit).\u001b[39;00m\n\u001b[1;32m    448\u001b[0m         \u001b[38;5;66;03m# Otherwise it looks like a bug in the code.\u001b[39;00m\n\u001b[1;32m    449\u001b[0m         six\u001b[38;5;241m.\u001b[39mraise_from(e, \u001b[38;5;28;01mNone\u001b[39;00m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:1377\u001b[0m, in \u001b[0;36mHTTPConnection.getresponse\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1375\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1376\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1377\u001b[0m         \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbegin\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1378\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m:\n\u001b[1;32m   1379\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mclose()\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:320\u001b[0m, in \u001b[0;36mHTTPResponse.begin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[38;5;66;03m# read until we get a non-100 response\u001b[39;00m\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 320\u001b[0m     version, status, reason \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_read_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    321\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m status \u001b[38;5;241m!=\u001b[39m CONTINUE:\n\u001b[1;32m    322\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/http/client.py:281\u001b[0m, in \u001b[0;36mHTTPResponse._read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_read_status\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m--> 281\u001b[0m     line \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_MAXLINE\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m)\u001b[49m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miso-8859-1\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    282\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(line) \u001b[38;5;241m>\u001b[39m _MAXLINE:\n\u001b[1;32m    283\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m LineTooLong(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus line\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/socket.py:704\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m    702\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m    703\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 704\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m    706\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1241\u001b[0m, in \u001b[0;36mSSLSocket.recv_into\u001b[0;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[1;32m   1237\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m flags \u001b[38;5;241m!=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m   1238\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m   1239\u001b[0m           \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnon-zero flags not allowed in calls to recv_into() on \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m\n\u001b[1;32m   1240\u001b[0m           \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__class__\u001b[39m)\n\u001b[0;32m-> 1241\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbytes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1242\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1243\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39mrecv_into(buffer, nbytes, flags)\n",
      "File \u001b[0;32m~/opt/anaconda3/lib/python3.9/ssl.py:1099\u001b[0m, in \u001b[0;36mSSLSocket.read\u001b[0;34m(self, len, buffer)\u001b[0m\n\u001b[1;32m   1097\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1098\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m buffer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m-> 1099\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sslobj\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mlen\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuffer\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1100\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1101\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_sslobj\u001b[38;5;241m.\u001b[39mread(\u001b[38;5;28mlen\u001b[39m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Step2: 调用chatgpt获得cot，每5次调用保存一下。\n",
    "注意：如果该行程序被终止，请务必从Step1开始重新执行\n",
    "\"\"\"\n",
    "call_num = 0\n",
    "for example in tqdm(instructgraph_test_data):\n",
    "    \n",
    "    task_name = example[\"task_name\"]\n",
    "    idx = example[\"idx\"]\n",
    "    \n",
    "    if \"{}-{}\".format(task_name, str(idx)) in saved_name:\n",
    "        continue\n",
    "    \n",
    "    answer = example[\"answer\"]\n",
    "    instruction = example[\"instruction\"]\n",
    "    \n",
    "    # 调用chatgpt获得推理结果\n",
    "    response = decoder_for_openai(\"gpt-4\", instruction)\n",
    "    call_num += 1\n",
    "    # 先生成，后续再单独判断推理的结果是否与标签一致\n",
    "    example[\"prediction\"] = response\n",
    "    prediction_example_list.append(example)\n",
    "    # 每5次成功调用就保存一次文件，避免丢失\n",
    "    if call_num % 5 == 0:\n",
    "#         with open(os.path.join(output_dir, \"all_predictions.json\"), \"w\", encoding=\"utf-8\") as fw:\n",
    "#             for example in prediction_example_list:\n",
    "#                 fw.write(json.dumps(example) + \"\\n\")\n",
    "        with open(os.path.join(output_dir, \"all_prediction.json\"), \"w\", encoding=\"utf-8\") as fw:\n",
    "            json.dump(prediction_example_list, fw, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "438216fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['graph-language-modeling-graph-node-cls-pubmed-1013',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1907',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-692',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1521',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-393',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1033',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-92',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-686',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-737',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1144',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1712',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-938',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1322',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1063',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-96',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1507',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-25',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1454',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1073',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1607',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-214',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-653',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-479',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1221',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-803',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-398',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-265',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1983',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-632',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-946',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1889',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1256',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1844',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1150',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1094',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1219',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-919',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-136',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1868',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1572',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-522',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-598',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1410',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1990',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1570',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-257',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1445',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1212',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1864',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-452',\n",
       " 'graph-language-modeling-graph-node-cls-pubmed-1412',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-376',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-224',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-652',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-212',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1251',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-276',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-636',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-515',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1281',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1969',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-697',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1083',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-385',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-676',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1138',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1561',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1396',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-406',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1938',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1750',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1701',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-963',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1437',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-99',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1685',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-213',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-883',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-678',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-245',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-755',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-741',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1773',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-538',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1143',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1641',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-569',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1488',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-628',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-699',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-500',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1018',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1734',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-993',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1786',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-589',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1650',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1805',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-892',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-168',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1658',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-wikipedia-1172',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-504',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-661',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-913',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-623',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-231',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-778',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-500',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-276',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-411',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-859',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-957',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-368',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-65',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-205',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-525',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-842',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-45',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-632',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-815',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-495',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-357',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-849',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-71',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-806',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-13',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-256',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-196',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-952',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-845',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-626',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-839',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-800',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-327',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-477',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-804',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-662',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-193',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-453',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-891',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-743',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-441',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-21',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-233',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-726',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-783',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-733',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-480',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-767',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-291',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-693',\n",
       " 'graph-language-modeling-graph-node-cls-citeseer-644',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1235',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1103',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3438',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2455',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-494',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1417',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3193',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3962',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3331',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2786',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-228',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1448',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1589',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3835',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3697',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2990',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3106',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-4073',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2549',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-748',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-300',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1149',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2085',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2732',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2445',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1353',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3280',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1935',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3600',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3083',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2057',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3278',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3409',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-918',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1777',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1793',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-499',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1683',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-3144',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1462',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-4380',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2946',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2940',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1344',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2462',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1069',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1821',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2567',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-1643',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-377',\n",
       " 'graph-language-modeling-graph-link-prediction-wikidata5m-2325',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-290',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-866',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1643',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-317',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1649',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-301',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-651',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-223',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-38',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1218',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-882',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1684',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1834',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1276',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-811',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1980',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1983',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-398',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1499',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1497',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-245',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1918',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-600',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-373',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1457',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1554',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-812',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1627',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-964',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-207',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-214',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-597',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-260',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-395',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1867',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1953',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-639',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-598',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1125',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1441',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-706',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1575',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-309',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-849',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1434',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1552',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-110',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-1253',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-556',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-841',\n",
       " 'graph-language-modeling-graph-caption-generation-webnlg-813',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-554',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-1081',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3322',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2869',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-660',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2192',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-1970',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2575',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3827',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4740',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-783',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2208',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5013',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-478',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2268',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4236',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5074',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5120',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-1633',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-1137',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5277',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4923',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4142',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3204',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2188',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2871',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2431',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5139',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-727',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3857',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2483',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-955',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-722',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4334',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4187',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5487',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-456',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2352',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3621',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-6314',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-6663',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2672',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4778',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-5724',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-4625',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2060',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-3334',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-238',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-1452',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2133',\n",
       " 'graph-language-modeling-graph-question-answering-grailqa-2282',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-181',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-530',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-319',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-533',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-844',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-958',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-600',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-945',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-788',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-624',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-245',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-42',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-363',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-744',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-867',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-407',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-161',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-403',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-894',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-645',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-193',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-710',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-752',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-236',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-981',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-987',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-392',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-938',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-215',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-74',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-615',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-44',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-436',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-629',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-512',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-8',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-380',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-497',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-734',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-870',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-612',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-699',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-414',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-868',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-230',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-994',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-61',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-546',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-292',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-267',\n",
       " 'graph-language-modeling-graph-question-answering-pathquestion-719',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-202',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-223',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-345',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-513',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-483',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-239',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-711',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-428',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-972',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-160',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-851',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-217',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-207',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-389',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-766',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-578',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-21',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-82',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-683',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-256',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-158',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-994',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-626',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-894',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-563',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-609',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-648',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-147',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-912',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-933',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-81',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-744',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-336',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-641',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-579',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-327',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-310',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-852',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-561',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-265',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-644',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-491',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-270',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-618',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-48',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-36',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-849',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-68',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-858',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-666',\n",
       " 'graph-language-modeling-graph-question-answering-wc2014-398',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1215',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-739',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1396',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1861',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1981',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-160',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-130',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1045',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1112',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-246',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-645',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-656',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1412',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1082',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1334',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1641',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1709',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-389',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1144',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1042',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-519',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-514',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-59',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-458',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1422',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-658',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1249',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1316',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1608',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-910',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1947',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1572',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1768',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1530',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-418',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-122',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1483',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-731',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1972',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1186',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1498',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-932',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1702',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-63',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1113',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-369',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1123',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1099',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1005',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-298',\n",
       " 'graph-language-modeling-graph-caption-generation-wikipedia-1851',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-133',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-896',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1807',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-6',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-249',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1561',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1253',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-279',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1448',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-232',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1965',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-834',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1848',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1352',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1817',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1954',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-796',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1449',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-265',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-163',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1380',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1205',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1493',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1150',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1507',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1203',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1250',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-53',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1044',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-857',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-723',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1036',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-924',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-627',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-894',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1321',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1560',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1299',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1741',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1677',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1101',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1699',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1054',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1576',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-59',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-674',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-642',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-1330',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-391',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-445',\n",
       " 'graph-language-modeling-graph-caption-generation-eventna-171',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-361',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-115',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-717',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-137',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-197',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-520',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-758',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-388',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-444',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-482',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-971',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-318',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-731',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-898',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-684',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-258',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-652',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-711',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-734',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-5',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-161',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-714',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-6',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-98',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-402',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-690',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-965',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-671',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-41',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-987',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-44',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-883',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-434',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-83',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-375',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-327',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-205',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-305',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-820',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-269',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-360',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-880',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-282',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-507',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-212',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-756',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-374',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-835',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-77',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-909',\n",
       " 'graph-language-modeling-graph-caption-generation-agenda-1',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-820',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-221',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-401',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-553',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-230',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-55',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-752',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-388',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-652',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-649',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-453',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-213',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-113',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-22',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-249',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-837',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-914',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-742',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-504',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-517',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-87',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-321',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-411',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-905',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-809',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-151',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-661',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-900',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-853',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-502',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-100',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-486',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-85',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-267',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-646',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-980',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-29',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-755',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-176',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-638',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-579',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-920',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-348',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-60',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-436',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-530',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-968',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-178',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-95',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-918',\n",
       " 'graph-language-modeling-graph-caption-generation-genwiki-301',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-188',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-153',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-775',\n",
       " 'graph-language-modeling-graph-relevance-inspection-648',\n",
       " 'graph-language-modeling-graph-relevance-inspection-383',\n",
       " 'graph-language-modeling-graph-relevance-inspection-369',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-86',\n",
       " 'graph-language-modeling-graph-relevance-inspection-47',\n",
       " 'graph-language-modeling-graph-relevance-inspection-484',\n",
       " 'graph-language-modeling-graph-relevance-inspection-523',\n",
       " 'graph-language-modeling-graph-relevance-inspection-392',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-373',\n",
       " 'graph-language-modeling-graph-relevance-inspection-414',\n",
       " 'graph-language-modeling-graph-relevance-inspection-857',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-426',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-430',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-791',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-239',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-395',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-763',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-174',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-266',\n",
       " 'graph-language-modeling-graph-relevance-inspection-487',\n",
       " 'graph-language-modeling-graph-relevance-inspection-883',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-language-modeling-graph-relevance-inspection-999',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1942',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-56',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-439',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2765',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1575',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2758',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-880',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1723',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2873',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2063',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2953',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1453',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2972',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-143',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-796',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2239',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2367',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-48',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2600',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1057',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2596',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1263',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2620',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2986',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1391',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1470',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2071',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2119',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2116',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1622',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2157',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2917',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1403',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1936',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2859',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1246',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2866',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2518',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-60',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2965',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1065',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1007',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1110',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2382',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-652',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1634',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1529',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2380',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-638',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-2845',\n",
       " 'graph-construction-modeling-knowledge-graph-generation-instructionuie-1702',\n",
       " 'graph-structure-modeling-degree-computing-35',\n",
       " 'graph-structure-modeling-degree-computing-155',\n",
       " 'graph-structure-modeling-degree-computing-9',\n",
       " 'graph-structure-modeling-degree-computing-319',\n",
       " 'graph-structure-modeling-degree-computing-205',\n",
       " 'graph-structure-modeling-degree-computing-213',\n",
       " 'graph-structure-modeling-degree-computing-328',\n",
       " 'graph-structure-modeling-degree-computing-165',\n",
       " 'graph-structure-modeling-degree-computing-246',\n",
       " 'graph-structure-modeling-degree-computing-130',\n",
       " 'graph-structure-modeling-degree-computing-90',\n",
       " 'graph-structure-modeling-degree-computing-1',\n",
       " 'graph-structure-modeling-degree-computing-188',\n",
       " 'graph-structure-modeling-degree-computing-95',\n",
       " 'graph-structure-modeling-degree-computing-100',\n",
       " 'graph-structure-modeling-degree-computing-353',\n",
       " 'graph-structure-modeling-degree-computing-20',\n",
       " 'graph-structure-modeling-degree-computing-7',\n",
       " 'graph-structure-modeling-degree-computing-51',\n",
       " 'graph-structure-modeling-degree-computing-137',\n",
       " 'graph-structure-modeling-degree-computing-265',\n",
       " 'graph-structure-modeling-degree-computing-196',\n",
       " 'graph-structure-modeling-degree-computing-306',\n",
       " 'graph-structure-modeling-degree-computing-166',\n",
       " 'graph-structure-modeling-degree-computing-310',\n",
       " 'graph-structure-modeling-degree-computing-314',\n",
       " 'graph-structure-modeling-degree-computing-199',\n",
       " 'graph-structure-modeling-degree-computing-82',\n",
       " 'graph-structure-modeling-degree-computing-192',\n",
       " 'graph-structure-modeling-degree-computing-281',\n",
       " 'graph-structure-modeling-degree-computing-48',\n",
       " 'graph-structure-modeling-degree-computing-164',\n",
       " 'graph-structure-modeling-degree-computing-239',\n",
       " 'graph-structure-modeling-degree-computing-96',\n",
       " 'graph-structure-modeling-degree-computing-2',\n",
       " 'graph-structure-modeling-degree-computing-236',\n",
       " 'graph-structure-modeling-degree-computing-30',\n",
       " 'graph-structure-modeling-degree-computing-288',\n",
       " 'graph-structure-modeling-degree-computing-129',\n",
       " 'graph-structure-modeling-degree-computing-167',\n",
       " 'graph-structure-modeling-degree-computing-104',\n",
       " 'graph-structure-modeling-degree-computing-332',\n",
       " 'graph-structure-modeling-degree-computing-282',\n",
       " 'graph-structure-modeling-degree-computing-283',\n",
       " 'graph-structure-modeling-degree-computing-120',\n",
       " 'graph-structure-modeling-degree-computing-121',\n",
       " 'graph-structure-modeling-degree-computing-173',\n",
       " 'graph-structure-modeling-degree-computing-72',\n",
       " 'graph-structure-modeling-degree-computing-64',\n",
       " 'graph-structure-modeling-degree-computing-170',\n",
       " 'graph-structure-modeling-degree-computing-11',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-91',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-75',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1079',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1596',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1451',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-822',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1482',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-675',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-580',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-705',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-231',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1664',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1106',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-553',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1501',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-560',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1161',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-19',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-894',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-765',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-272',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1399',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-193',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-16',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-937',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1178',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1158',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-439',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1135',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-923',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-988',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1395',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1440',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-708',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-921',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-778',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1293',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1467',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1129',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1369',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-669',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-804',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1263',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1204',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1396',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1316',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1670',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1020',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1403',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-481',\n",
       " 'graph-language-modeling-graph-node-cls-ogbn-products-1262',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1024',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-888',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1722',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1493',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1757',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-917',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-34',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1511',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-918',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-170',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1471',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1170',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1041',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-935',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1484',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1011',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1659',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-548',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1626',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1456',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-662',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-99',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1228',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1079',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-413',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1620',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1568',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1581',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-517',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1920',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-940',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1175',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1142',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-706',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-48',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1135',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-925',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1424',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1179',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1900',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1396',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-285',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1574',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-313',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1744',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1681',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-903',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-393',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-1753',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-796',\n",
       " 'graph-thought-modeling-natural-language-reasoning-nlpreasoning-807',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-338',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-74',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-260',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-515',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-172',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-69',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-711',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-169',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-258',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-75',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-393',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-458',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-429',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-520',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-26',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-271',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-397',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-304',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-577',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-633',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-694',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-278',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-514',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-126',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-734',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-296',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-469',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-682',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-242',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-197',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-400',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-93',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-295',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-488',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-81',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-465',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-641',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-245',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-193',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-124',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-43',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-612',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-471',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-695',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-314',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-106',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-568',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-255',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-571',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-599',\n",
       " 'graph-language-modeling-graph-link-prediction-conceptnet-409',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-228',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-92',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-111',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1887',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1280',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-536',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-893',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-397',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-996',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1157',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-852',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1525',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-602',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-283',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-704',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1235',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-21',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1184',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1390',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1465',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1477',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1729',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-376',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1155',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-902',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-883',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1558',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1569',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1260',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-611',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1253',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1349',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1545',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1520',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1864',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1877',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-715',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-474',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-658',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-713',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1212',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1196',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1804',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-986',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-961',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1383',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1011',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1749',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1647',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1598',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-moivelens-1888',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-781',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-758',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1762',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1673',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-586',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-209',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1800',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-597',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-554',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1254',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-71',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-148',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1184',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1836',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1517',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-391',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1304',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1875',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1468',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-385',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1552',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-315',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-797',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1530',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-153',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1888',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-600',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1855',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-439',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1548',\n",
       " 'graph-language-modeling-graph-collaboration-filtering-lastfm-1861',\n",
       " ...]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, update_display\n",
    "display(prediction_example_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "23b4cc4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'graph-construction-modeling-structure-graph-generation-directedweighted'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
